#!/usr/bin/env python  
#coding=utf-8

import urllib2, re, os, time
import sqlite3
from BeautifulSoup import BeautifulSoup

##函数
#解析页面函数
#getpage
def getPage(url):
  try:
    page = urllib2.urlopen(url) 
    soup = BeautifulSoup(page) 
    return soup
  except:
    return 0

#去掉HTML标签
def dropTags(soupStr):
  soupStr = str(soupStr)
  soupStr = re.compile('<[^>]+>').sub("",soupStr)
  soupStr = re.compile('&nbsp;').sub("",soupStr)
  soupStr = re.compile('&copy;').sub("",soupStr)
  return re.compile('豆瓣').sub("",soupStr)

  
##Begin！
#设置当前目录为工作目录
os.chdir(os.path.dirname(__file__))

#数据库连接
conn = sqlite3.connect('doubanMoviesDB.db')
curs = conn.cursor()

#豆瓣电影网址
webUrl = 'http://movie.douban.com/tag/'

webTags =['爱情','喜剧','动画','经典','科幻','动作','青春','剧情','悬疑','惊悚','纪录片','励志','恐怖','犯罪','魔幻','战争','搞笑','文艺','动画短片','短片','情色','黑色幽默','童年','传记','同志','音乐','暴力','黑帮','女性','浪漫','感人','家庭','童话','史诗','cult','震撼']

totalNum = 0

for webTag in webTags:
  
  print webTag
  
  num = 0
  
  while num <= 10:
    
    webTagUrl = webUrl + urllib2.quote(webTag) + "?start="+str(num*20)+"&type=T"
    
    print webTagUrl
    
    time.sleep(2)
    
    webTagSoup = getPage(webTagUrl)
    
    webSubjectListSoups = webTagSoup.find('div', {'id':'subject_list'}).findAll('div', {'class':'pl2'})
    
    for webSubjectListSoup in webSubjectListSoups:
      
      webSubjectUrl = webSubjectListSoup.find('a')['href']
      
      print webSubjectUrl
      
      subjectId = str(webSubjectUrl)[-8:-1]
      
      print subjectId
      
      checkquery = 'select subjectid from movies where subjectid =' +str(subjectId)
      curs.execute(checkquery)
      
      if curs.fetchone() != None:
        
        print '有了已经！！！'
        
        continue
      
      else:
        
        time.sleep(2)
        
        subjectSoup = getPage(webSubjectUrl)
        
        if subjectSoup == 0:
          
          continue
          
        elif subjectSoup.find('span', {'property':'v:summary'}):
          
          h1Span = subjectSoup.find('span', {'property':'v:itemreviewed'})
          
          movieName = h1Span.string

          print movieName

          if subjectSoup.find('span', {'class':'all hidden'}):
            soupStr = str(subjectSoup.find('span', {'class':'all hidden'}))
            description = dropTags(soupStr)
            print description
            
          else:
            soupStr = str(subjectSoup.find('span', {'property':'v:summary'}))
            description = dropTags(soupStr)
            print description
          
          query = 'insert into movies(name,subjectid,description,tag) values ("'+ movieName +'","' + str(subjectId) +'","'+ description +'","'+ webTag +'")'
          
          print query
          curs.execute(query)
          conn.commit()
          
          totalNum +=1
        
        else:
          continue
    
    num +=1
    
conn.close()
    




